const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;

const mongo_url = "mongodb://egg_cnode:bojun1234@124.223.34.189:27127/egg_cnode?authSource=egg_cnode";


const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  browser = await puppeteer.launch({
    headless: false,
    ignoreHTTPSErrors: true,
    args: ['--no-sandbox', `--proxy-server=http://u5329.b5.t.16yun.cn:6460`]
  });
  const page = (await browser.pages())[0];
  await page.authenticate({'username': '16GADNHJ', 'password': '936441'});
  await page.setViewport({width: 1500, height: 0});

  const client = await MongoClient.connect(mongo_url, {useNewUrlParser: true, useUnifiedTopology: true});
  const db = client.db('egg_cnode');

  let sub_url = '';
  let tab = 'knowledge';
  let sub_tab = '果壳';
  const fileName = `${tab}_${sub_tab}.json`;
  const source = '果壳' + (new Date(+new Date() + 1000 * 3600 * 8)).toISOString().split('T')[0];
  const real_author_name = '科学人';


  page.on('pageerror', msg => {
    console.log(msg);
  });

  let url = `https://www.guokr.com/science/category/all`;
  await page.goto(url, {waitUntil: 'domcontentloaded'});


  await page.waitForSelector('.ePjesp');

  let preScrollHeight = 0;
  let scrollHeight = -1;
  let cnt = 0;
  let retry_cnt = 0;
  const SLEEP_TIME = 1000;
  while (cnt < 1000) {
    let totalTopics = [];
    // 详情信息是根据滚动异步加载，所以需要让页面滚动到屏幕最下方，通过延迟等待的方式进行多次滚动
    let scrollH1 = await page.evaluate(async () => {
      let h1 = document.body.scrollHeight;
      window.scrollTo(0, h1);
      return h1;
    });
    // 延迟等待时间
    await page.waitForTimeout(SLEEP_TIME);
    let scrollH2 = await page.evaluate(async () => {
      return document.body.scrollHeight;
    });
    let scrollResult = [scrollH1, scrollH2];
    preScrollHeight = scrollResult[0];
    scrollHeight = scrollResult[1];


    await page.click('.ePjesp');
    await page.waitForTimeout(1000);
    console.log(scrollResult, cnt++);

    if (cnt < 200) {
      continue;
    }


    const eleHandles = await page.$$('.panel-article');

    for (let i = cnt * 10; i < eleHandles.length; i++) {
      let newPage;
      try {
        const eleHandle = eleHandles[i];
        const title = await eleHandle.$eval('.eCquRY', el => el.innerText);
        const intro = await eleHandle.$eval('.kQfPNg', el => el.innerText);
        const pic = await eleHandle.$eval('.jjAsJJ > img', el => el.src);
        const newPagePromise = new Promise(res =>
          browser.once('targetcreated',
            target => res(target.page())
          )
        );
        await eleHandle.click();
        newPage = await newPagePromise;

        const content = await newPage.url();
        let topic = {
          title,
          content,
          original_url: content,
          tab: 'knowledge',
          sub_tab: '果壳',
          source,
          intro,
          pic,
          real_author_name,
          author_id: '61dda2a097ceba9ea3ea8ca2',
          author_name: '果壳',
          author_avatar: '/public/images/guokr.png',
          is_public: true,
          deleted: false,
          create_at: new Date(),
          update_at: new Date(),
          last_reply_at: new Date(),
          top: false,
        }

        totalTopics.push(topic);

      } catch (e) {
        console.error(e);
      } finally {
        if (newPage) {
          await newPage.close();
        }
      }
    }

    totalTopics = totalTopics.slice(-10);
    try {
      let result = await db.collection('topics').insertMany(totalTopics, {ordered: false});
      console.log(result.result, cnt);
    } catch (e) {
      console.error(e);
    }
  }
};


main();


